/*!
 *	Crawler
 */
import java.net.URLEncoder
import groovy.time.TimeCategory
import groovy.json.JsonSlurper
import groovy.util.XmlSlurper
import groovyx.gpars.GParsPool
import sun.misc.BASE64Decoder
import org.jsoup.Jsoup
import org.jsoup.Connection.Method


ebook = [
	'1': ['莽荒记', 'http://m.baidu.com/tc?srd=1&appui=alaxs&ajax=1&alalog=1&gid=3961103225&pageType=router&src=http%3A%2F%2Fwww.binhuo.com%2Fhtml%2F35%2F35744%2Findex.html&dir=1&ssid=0&from=844b&uid=0&pu=usm%401%2Csz%401320_1001%2Cta%40iphone_2_4.2_3_537&bd_page_type=1&baiduid=E7C0CD608B2134E4DDC9B549AAD41CDF&tj=wise_novel_book_1_0_10_title&id=wisenovel&logid=1412693686724156&hasRp=true&time=&skey='],
	'2': ['网游之狂兽逆天', 'http://m.baidu.com/tc?srd=1&appui=alaxs&ajax=1&alalog=1&gid=1954128079&pageType=router&src=http%3A%2F%2Fwww.jdxs.net%2Ffiles%2Farticle%2Fhtml%2F65%2F65858%2Findex.html&dir=1&ssid=0&from=844b&uid=0&pu=usm%400%2Csz%401320_1001%2Cta%40iphone_2_4.2_3_537&bd_page_type=1&baiduid=E7C0CD608B2134E4DDC9B549AAD41CDF&tj=wise_novel_book_1_0_10_title&id=wisenovel&logid=1412692625011768&hasRp=true&time=&skey='],
	'3': ['蜗居', 'http://m.baidu.com/tc?srd=1&appui=alaxs&ajax=1&alalog=1&gid=1822469960&pageType=router&src=http%3A%2F%2Fwww.nv001.cn%2Ffiles%2Farticle%2Fhtml%2F3%2F3415%2Findex.html&dir=1&loading=true&ssid=0&from=844b&uid=0&pu=usm%401%2Csz%401320_2001%2Cta%40iphone_1_8.0_3_600&bd_page_type=1&baiduid=B5D51C23015339205CEC21789F5318FA&tj=wise_novel_book_3_0_10_l1&id=wisenovel&logid=1421464699290803&hasRp=true&time=&skey='],
	'4': ['异世灵武天下', 'http://m.baidu.com/tc?srd=1&appui=alaxs&ajax=1&alalog=1&gid=1705173889&pageType=router&src=http%3A%2F%2Fwww.chkee.com%2Fchkbook%2F14%2F14218%2Findex.html&dir=1&loading=true&ssid=0&from=844b&uid=0&pu=sz%401320_2001%2Cta%40iphone_1_8.0_3_600%2Cusm%400&bd_page_type=1&baiduid=CA1D5A969993BBD7AE8D830E8F95F621&tj=wise_novel_book_1_0_10_l1&id=wisenovel&logid=1436014870728214&hasRp=true&time=&skey=']
]

// default action
ebookaction = ebook['1'][1]
if (this.args.length) {
	ebookaction = ebook[this.args[0]][1]
}




debug = 0
if (debug == 1) {
	println "setting proxy -> proxy.pvgl.sap.corp:8080"
	System.setProperty("http.proxyHost", "proxy.pvgl.sap.corp")
	System.setProperty("http.proxyPort", "8080")
	System.setProperty("https.proxyHost", "proxy.pvgl.sap.corp")
	System.setProperty("https.proxyPort", "8080")
} else if (debug == 2) {
	println "setting proxy -> localhost:8888"
	System.setProperty("http.proxyHost", "localhost")
	System.setProperty("http.proxyPort", "8888")
	System.setProperty("https.proxyHost", "localhost")
	System.setProperty("https.proxyPort", "8888")
}


DEFAULT_TIMEOUT = 30000

def fetchContents() {
	def action = ebookaction
	def content = new JsonSlurper().parse(new URL(action), "utf-8")
	return content;
}

def fetchArticles(content) {
	def slurper = new JsonSlurper()
	return GParsPool.withPool(32) {
		content.data.group.collectParallel {
			println " retreiving ${it.href}"
			def cid = URLEncoder.encode(it.cid)
			def href = URLEncoder.encode(it.href)
			def action = "http://m.baidu.com/tc?srd=1&appui=alaxs&ajax=1&gid=1954128079&alals=1&preNum=1&preLoad=true&src=${href}&cid=${cid}&time=&skey=&id=wisenovel"
			//println " cc..."
			def article = slurper.parse(new URL(action), "utf-8")
			//println " cc: ${article}"
			return [href: it.href, title: it.text, article: article]
		}
	}.each {
		println " processing ${it.href}"
		//it.content = slurper.parse(it.article, "utf-8").content
		try {
			def cc = it.article.data.first().content.replace('<br/>','__BR__')
			cc = Jsoup.parse(cc).text().replace('__BR__', '\n')
			it.content = cc
		}
		catch(Exception e) {
			println " failed to process ${it.article}"
			it.content = "BAD_CONTENT"
		}
	}
}

def save(content, articles) {
	new File(content.data.title + '.txt').withWriter("unicode") { fs ->
		fs.println "[${content.data.title}]"
		fs.println "作者: ${content.data.author}"
		fs.println ""

		articles.each {
			fs.println "[${it.title}]"
			fs.println it.content
			fs.println ""
		}
	}
}

def main() {
	def start = new Date()

	println "crawler is working..."

	def content = fetchContents()
	println "fetch content, src=${content.data.src}"

	def articles = fetchArticles(content)
	println "fetch articles successfully"

	save(content, articles)
	println "save ebook successfully"

	def end = new Date()
	def elapsed = TimeCategory.minus(end, start)
	println "all done, time elapsed: ${elapsed}"
}

main()